/*
Copyright (C) 2003-2004 Pierrick Brihaye
pierrick.brihaye@wanadoo.fr
 
Original Perl code :
Portions (c) 2002 QAMUS LLC (www.qamus.org), 
(c) 2002 Trustees of the University of Pennsylvania 
 
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; either version 2
of the License, or (at your option) any later version.

This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
See the GNU General Public License for more details.

You should have received a copy of the GNU General Public License
along with this program; if not, write to the
Free Software Foundation, Inc.
59 Temple Place - Suite 330, Boston, MA  02111-1307, USA
or connect to:
http://www.fsf.org/copyleft/gpl.html
*/

package gpl.pierrick.brihaye.aramorph;

import gpl.pierrick.brihaye.util.LRUCache;
import gpl.pierrick.brihaye.util.Pair;
import gpl.pierrick.brihaye.util.RangeSet;
import gpl.pierrick.brihaye.util.StringOperations;

import java.io.BufferedReader;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.LineNumberReader;
import java.io.PrintStream;
import java.io.UnsupportedEncodingException;
import java.nio.charset.Charset;
import java.nio.charset.IllegalCharsetNameException;
import java.nio.charset.UnsupportedCharsetException;
import java.text.DecimalFormat;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.Map;
import java.util.Set;
import java.util.logging.Handler;
import java.util.logging.Level;
import java.util.logging.Logger;
import java.util.logging.SimpleFormatter;
import java.util.logging.StreamHandler;

/** A java port of Buckwalter Arabic Morphological Analyzer Version 1.0.
 * Original Perl distribution avalaible from :
 * <a
 * href="http://www.ldc.upenn.edu/Catalog/CatalogEntry.jsp?catalogId=LDC2002L49">LDC
 * Catalog</a>
 * @author Pierrick Brihaye, 2003
 * @author Ahmed Saad, 2005
 */
public class AraMorph {
	/** A logger used to write log messages */
	private static final Logger logger = Logger.getLogger(AraMorph.class.getName());

	/** Size of cache that holds solutions of found tokens */
	private static final int FOUND_LIMIT = 200;

	/** Size of cache that holds not found tokens */
	private static final int NOT_FOUND_LIMIT = 50;

	/** A set that holds arabic letter */
	public static final Collection<Integer> ARABIC_LETTERS;

	/**Alternative spellings*/
	private static final Set<Pair> ALTERNATIVE_SPELLINGS;
	/**Alternative endings*/
	private static final Set<Pair> ALTERNATIVE_ENDINGS;
	
	/**Maps each Arabic letter to its Roman transliteration*/
	private static final Map<Character, Object> ARABIC2ROMAN;
	
	/**Maps each Roman letter to its Arabic transliteration*/
	private static final Map<Character, Object> ROMAN2ARABIC;

	/** An instance of AraMorph that has its dictionaris normalized and uses JDBC */
	private static AraMorph NORMALIZED_JDBC_ARAMORPH;
	
	/** The dictionary handler. */
	private DictionaryHandler dict;

	/** The solutions handler. */
	protected SolutionsHandler sol = null;

	//Stats
	
	/** Lines processed */
	private int linesCounter = 0;
	/** Arabic tokens processed */
	private int arabicTokensCounter = 0;
	/** Not arabic tokens processed */
	private int nonArabicTokensCounter = 0;
	
	/** Arabic words which have been succesfully analyzed.
	 * <PRE>key</PRE> = word
	 * <PRE>value</PRE> = occurences
	 */
	private Map<String,Integer> found = new LRUCache<String,Integer>(FOUND_LIMIT);
	
	/** Arabic words which have not been succesfully analyzed.
	 * <PRE>key</PRE> = word
	 * <PRE>value</PRE> = occurences
	 */
	private Map<String,Integer> notFound = new LRUCache<String,Integer>(NOT_FOUND_LIMIT);
	
	//Arabic letters
	public static final char ARABIC_LETTER_HAMZA = '\u0621';
	public static final char ARABIC_LETTER_ALEF_WITH_MADDA_ABOVE = '\u0622';
	public static final char ARABIC_LETTER_ALEF_WITH_HAMZA_ABOVE = '\u0623';
	public static final char ARABIC_LETTER_WAW_WITH_HAMZA_ABOVE = '\u0624';
	public static final char ARABIC_LETTER_ALEF_WITH_HAMZA_BELOW = '\u0625';
	public static final char ARABIC_LETTER_YEH_WITH_HAMZA_ABOVE = '\u0626';
	public static final char ARABIC_LETTER_ALEF = '\u0627';
	public static final char ARABIC_LETTER_BEH = '\u0628';
	public static final char ARABIC_LETTER_TEH_MARBUTA = '\u0629';
	public static final char ARABIC_LETTER_TEH = '\u062A';
	public static final char ARABIC_LETTER_THEH = '\u062B';
	public static final char ARABIC_LETTER_JEEM = '\u062C';
	public static final char ARABIC_LETTER_HAH = '\u062D';
	public static final char ARABIC_LETTER_KHAH = '\u062E';
	public static final char ARABIC_LETTER_DAL = '\u062F';
	public static final char ARABIC_LETTER_THAL = '\u0630';
	public static final char ARABIC_LETTER_REH = '\u0631';
	public static final char ARABIC_LETTER_ZAY = '\u0632';
	public static final char ARABIC_LETTER_SEEN = '\u0633';
	public static final char ARABIC_LETTER_SHEEN = '\u0634';
	public static final char ARABIC_LETTER_SAD = '\u0635';
	public static final char ARABIC_LETTER_DAD = '\u0636';
	public static final char ARABIC_LETTER_TAH = '\u0637';
	public static final char ARABIC_LETTER_ZAH = '\u0638';
	public static final char ARABIC_LETTER_AIN = '\u0639';
	public static final char ARABIC_LETTER_GHAIN = '\u063A';
	public static final char ARABIC_KASHIDA = '\u0640';
	public static final char ARABIC_LETTER_FEH = '\u0641';
	public static final char ARABIC_LETTER_QAF = '\u0642';
	public static final char ARABIC_LETTER_KAF = '\u0643';
	public static final char ARABIC_LETTER_LAM = '\u0644';
	public static final char ARABIC_LETTER_MEEM = '\u0645';
	public static final char ARABIC_LETTER_NOON = '\u0646';
	public static final char ARABIC_LETTER_HEH = '\u0647';
	public static final char ARABIC_LETTER_WAW = '\u0648';
	public static final char ARABIC_LETTER_ALEF_MAKSOORA = '\u0649';
	public static final char ARABIC_LETTER_YEH = '\u064A';
	public static final char ARABIC_TANWEEN_FATH = '\u064B';
	public static final char ARABIC_TANWEEN_DAM = '\u064C';
	public static final char ARABIC_TANWEEN_KASR = '\u064D';
	public static final char ARABIC_FATHA = '\u064E';
	public static final char ARABIC_DAMMA = '\u064F';
	public static final char ARABIC_KASRA = '\u0650';
	public static final char ARABIC_SHADDA = '\u0651';
	public static final char ARABIC_SUKUN = '\u0652';
	public static final char ARABIC_LETTER_SUPERSCRIPT_ALEF = '\u0670';
	public static final char ARABIC_LETTER_ALEF_WASLA = '\u0671';
	public static final char ARABIC_COMMA = '\u060C';
	public static final char ARABIC_SEMICOLON = '\u061B';
	public static final char ARABIC_QUESTION_MARK = '\u061F';
	public static final char ARABIC_LETTER_PEH = '\u067E';
	public static final char ARABIC_LETTER_TCHEH = '\u0686';
	public static final char ARABIC_LETTER_VEH = '\u06A4';
	public static final char ARABIC_LETTER_GAF = '\u06AF';
	public static final char ARABIC_LETTER_JEH = '\u0698';
	
	/**
	 * Intitializes static members
	 */
	static {
		Logger.getLogger("gpl.pierrick.brihaye.aramorph").setLevel(Level.WARNING);
		//intitalize arabic letters
		Collection<Integer> arabicLetters = new RangeSet();
		arabicLetters.add((int)'\u067E');
		arabicLetters.add((int)'\u0686');
		arabicLetters.add((int)'\u0698');
		arabicLetters.add((int)'\u06AF');
		arabicLetters.addAll(new RangeSet('\u0621','\u063A'));
		arabicLetters.addAll(new RangeSet('\u0641','\u0652'));
		ARABIC_LETTERS = Collections.unmodifiableCollection(arabicLetters);

		//initialize Roman to Arabic map
		ROMAN2ARABIC = new HashMap<Character,Object>();
		ROMAN2ARABIC.put('\'', ARABIC_LETTER_HAMZA);
		ROMAN2ARABIC.put('|', ARABIC_LETTER_ALEF_WITH_MADDA_ABOVE);
		ROMAN2ARABIC.put('>', ARABIC_LETTER_ALEF_WITH_HAMZA_ABOVE);
		ROMAN2ARABIC.put('&', ARABIC_LETTER_WAW_WITH_HAMZA_ABOVE);
		ROMAN2ARABIC.put('<', ARABIC_LETTER_ALEF_WITH_HAMZA_BELOW);
		ROMAN2ARABIC.put('}', ARABIC_LETTER_YEH_WITH_HAMZA_ABOVE);
		ROMAN2ARABIC.put('A', ARABIC_LETTER_ALEF);
		ROMAN2ARABIC.put('b', ARABIC_LETTER_BEH);
		ROMAN2ARABIC.put('p', ARABIC_LETTER_TEH_MARBUTA);
		ROMAN2ARABIC.put('t', ARABIC_LETTER_TEH);
		ROMAN2ARABIC.put('v', ARABIC_LETTER_THEH);
		ROMAN2ARABIC.put('j', ARABIC_LETTER_JEEM);
		ROMAN2ARABIC.put('H', ARABIC_LETTER_HAH);
		ROMAN2ARABIC.put('x', ARABIC_LETTER_KHAH);
		ROMAN2ARABIC.put('d', ARABIC_LETTER_DAL);
		ROMAN2ARABIC.put('*', ARABIC_LETTER_THAL);
		ROMAN2ARABIC.put('r', ARABIC_LETTER_REH);
		ROMAN2ARABIC.put('z', ARABIC_LETTER_ZAY);
		ROMAN2ARABIC.put('s', ARABIC_LETTER_SEEN);
		ROMAN2ARABIC.put('$', ARABIC_LETTER_SHEEN);
		ROMAN2ARABIC.put('S', ARABIC_LETTER_SAD);
		ROMAN2ARABIC.put('D', ARABIC_LETTER_DAD);
		ROMAN2ARABIC.put('T', ARABIC_LETTER_TAH);
		ROMAN2ARABIC.put('Z', ARABIC_LETTER_ZAH);
		ROMAN2ARABIC.put('E', ARABIC_LETTER_AIN);
		ROMAN2ARABIC.put('g', ARABIC_LETTER_GHAIN);
		ROMAN2ARABIC.put('_', ARABIC_KASHIDA);
		ROMAN2ARABIC.put('f', ARABIC_LETTER_FEH);
		ROMAN2ARABIC.put('q', ARABIC_LETTER_QAF);
		ROMAN2ARABIC.put('k', ARABIC_LETTER_KAF);
		ROMAN2ARABIC.put('l', ARABIC_LETTER_LAM);
		ROMAN2ARABIC.put('m', ARABIC_LETTER_MEEM);
		ROMAN2ARABIC.put('n', ARABIC_LETTER_NOON);
		ROMAN2ARABIC.put('h', ARABIC_LETTER_HEH);
		ROMAN2ARABIC.put('w', ARABIC_LETTER_WAW);
		ROMAN2ARABIC.put('Y', ARABIC_LETTER_ALEF_MAKSOORA);
		ROMAN2ARABIC.put('y', ARABIC_LETTER_YEH);
		ROMAN2ARABIC.put('F', ARABIC_TANWEEN_FATH);
		ROMAN2ARABIC.put('N', ARABIC_TANWEEN_DAM);
		ROMAN2ARABIC.put('K', ARABIC_TANWEEN_KASR);
		ROMAN2ARABIC.put('a', ARABIC_FATHA);
		ROMAN2ARABIC.put('u', ARABIC_DAMMA);
		ROMAN2ARABIC.put('i', ARABIC_KASRA);
		ROMAN2ARABIC.put('~', ARABIC_SHADDA);
		ROMAN2ARABIC.put('o', ARABIC_SUKUN);
		ROMAN2ARABIC.put('`', ARABIC_LETTER_SUPERSCRIPT_ALEF);
		ROMAN2ARABIC.put('{', ARABIC_LETTER_ALEF_WASLA);
		ROMAN2ARABIC.put('P', ARABIC_LETTER_PEH);
		ROMAN2ARABIC.put('J', ARABIC_LETTER_TCHEH);
		ROMAN2ARABIC.put('V', ARABIC_LETTER_VEH);
		ROMAN2ARABIC.put('G', ARABIC_LETTER_GAF);
		ROMAN2ARABIC.put('R', ARABIC_LETTER_JEH); //(no more in Buckwalter system)
		//Not in Buckwalter system \u0679 : ARABIC LETTER TTEH);
		//Not in Buckwalter system \u0688 : ARABIC LETTER DDAL);
		//Not in Buckwalter system \u06A9 : ARABIC LETTER KEHEH);
		//Not in Buckwalter system \u0691 : ARABIC LETTER RREH);
		//Not in Buckwalter system \u06BA : ARABIC LETTER NOON GHUNNA);
		//Not in Buckwalter system \u06BE : ARABIC LETTER HEH DOACHASHMEE);
		//Not in Buckwalter system \u06C1 : ARABIC LETTER HEH GOAL);
		//Not in Buckwalter system \u06D2 : ARABIC LETTER YEH BARREE);
		ROMAN2ARABIC.put(',', ARABIC_COMMA);
		ROMAN2ARABIC.put(';', ARABIC_SEMICOLON);
		ROMAN2ARABIC.put('?',ARABIC_QUESTION_MARK);
		
		//initialize Arabic to Roman map
		ARABIC2ROMAN = new HashMap<Character,Object>();
		ARABIC2ROMAN.put(ARABIC_LETTER_HAMZA, '\'');
		ARABIC2ROMAN.put(ARABIC_LETTER_ALEF_WITH_MADDA_ABOVE, '|'); 
		ARABIC2ROMAN.put(ARABIC_LETTER_ALEF_WITH_HAMZA_ABOVE, '>'); 
		ARABIC2ROMAN.put(ARABIC_LETTER_WAW_WITH_HAMZA_ABOVE, '&');
		ARABIC2ROMAN.put(ARABIC_LETTER_ALEF_WITH_HAMZA_BELOW, '<');
		ARABIC2ROMAN.put(ARABIC_LETTER_YEH_WITH_HAMZA_ABOVE, '}');
		ARABIC2ROMAN.put(ARABIC_LETTER_ALEF, 'A');
		ARABIC2ROMAN.put(ARABIC_LETTER_BEH, 'b');
		ARABIC2ROMAN.put(ARABIC_LETTER_TEH_MARBUTA, 'p');
		ARABIC2ROMAN.put(ARABIC_LETTER_TEH, 't');
		ARABIC2ROMAN.put(ARABIC_LETTER_THEH, 'v');
		ARABIC2ROMAN.put(ARABIC_LETTER_JEEM, 'j');
		ARABIC2ROMAN.put(ARABIC_LETTER_HAH, 'H');
		ARABIC2ROMAN.put(ARABIC_LETTER_KHAH, 'x');
		ARABIC2ROMAN.put(ARABIC_LETTER_DAL, 'd');
		ARABIC2ROMAN.put(ARABIC_LETTER_THAL, '*');
		ARABIC2ROMAN.put(ARABIC_LETTER_REH, 'r');
		ARABIC2ROMAN.put(ARABIC_LETTER_ZAY, 'z');
		ARABIC2ROMAN.put(ARABIC_LETTER_SEEN, 's');
		ARABIC2ROMAN.put(ARABIC_LETTER_SHEEN, '$');
		ARABIC2ROMAN.put(ARABIC_LETTER_SAD, 'S');
		ARABIC2ROMAN.put(ARABIC_LETTER_DAD, 'D');
		ARABIC2ROMAN.put(ARABIC_LETTER_TAH, 'T');
		ARABIC2ROMAN.put(ARABIC_LETTER_ZAH, 'Z');
		ARABIC2ROMAN.put(ARABIC_LETTER_AIN, 'E');
		ARABIC2ROMAN.put(ARABIC_LETTER_GHAIN, 'g');
		ARABIC2ROMAN.put(ARABIC_KASHIDA, '_');
		ARABIC2ROMAN.put(ARABIC_LETTER_FEH, 'f');
		ARABIC2ROMAN.put(ARABIC_LETTER_QAF, 'q');
		ARABIC2ROMAN.put(ARABIC_LETTER_KAF, 'k');
		ARABIC2ROMAN.put(ARABIC_LETTER_LAM, 'l');
		ARABIC2ROMAN.put(ARABIC_LETTER_MEEM, 'm');
		ARABIC2ROMAN.put(ARABIC_LETTER_NOON, 'n');
		ARABIC2ROMAN.put(ARABIC_LETTER_HEH, 'h');
		ARABIC2ROMAN.put(ARABIC_LETTER_WAW, 'w');
		ARABIC2ROMAN.put(ARABIC_LETTER_ALEF_MAKSOORA, 'Y');
		ARABIC2ROMAN.put(ARABIC_LETTER_YEH, 'y');
		//Not suitable for morphological analysis : remove all vowels/diacritics, i.e. undo the job !
		ARABIC2ROMAN.put(ARABIC_TANWEEN_FATH, '\0');
		ARABIC2ROMAN.put(ARABIC_TANWEEN_DAM, '\0');
		ARABIC2ROMAN.put(ARABIC_TANWEEN_KASR, '\0');
		ARABIC2ROMAN.put(ARABIC_FATHA, '\0');
		ARABIC2ROMAN.put(ARABIC_DAMMA, '\0');
		ARABIC2ROMAN.put(ARABIC_KASRA, '\0');
		ARABIC2ROMAN.put(ARABIC_SHADDA, '\0');
		ARABIC2ROMAN.put(ARABIC_SUKUN, '\0');	
		//TODO: how to handle ARABIC LETTER SUPERSCRIPT ALEF and ARABIC LETTER ALEF WASLA ?		
		ARABIC2ROMAN.put(ARABIC_LETTER_SUPERSCRIPT_ALEF, '\0');
		ARABIC2ROMAN.put(ARABIC_LETTER_ALEF_WASLA, '\0');
		
		ARABIC2ROMAN.put(ARABIC_LETTER_PEH, 'P');
		ARABIC2ROMAN.put(ARABIC_LETTER_TCHEH, 'J');
		ARABIC2ROMAN.put(ARABIC_LETTER_VEH, 'V');
		ARABIC2ROMAN.put(ARABIC_LETTER_GAF, 'G');
		ARABIC2ROMAN.put(ARABIC_LETTER_JEH, 'R'); //no_more_in_Buckwalter_system)
		//Not in Buckwalter system \u0679 : ARABIC LETTER TTEH
		//Not in Buckwalter system \u0688 : ARABIC LETTER DDAL
		//Not in Buckwalter system \u06A9 : ARABIC LETTER KEHEH
		//Not in Buckwalter system \u0691 : ARABIC LETTER RREH
		//Not in Buckwalter system \u06BA : ARABIC LETTER NOON GHUNNA
		//Not in Buckwalter system \u06BE : ARABIC LETTER HEH DOACHASHMEE
		//Not in Buckwalter system \u06C1 : ARABIC LETTER HEH GOAL
		//Not in Buckwalter system \u06D2 : ARABIC LETTER YEH BARREE
		ARABIC2ROMAN.put(ARABIC_COMMA, ',');
		ARABIC2ROMAN.put(ARABIC_SEMICOLON, ';');
		ARABIC2ROMAN.put(ARABIC_QUESTION_MARK, '?');
		//Not significant for morphological analysis
		ARABIC2ROMAN.put(ARABIC_KASHIDA, '\0');

		//initialize alternative spellings
		ALTERNATIVE_SPELLINGS = new HashSet<Pair>();
		ALTERNATIVE_SPELLINGS.add(new Pair(""+ARABIC_LETTER_WAW+ARABIC_LETTER_HAMZA, ""+ARABIC_LETTER_WAW_WITH_HAMZA_ABOVE));
		ALTERNATIVE_SPELLINGS.add(new Pair(""+ARABIC_LETTER_WAW_WITH_HAMZA_ABOVE, ""+ARABIC_LETTER_WAW+ARABIC_LETTER_HAMZA));
		
		//initialize alternative word endings
		ALTERNATIVE_ENDINGS = new HashSet<Pair>();
		ALTERNATIVE_ENDINGS.add(new Pair(""+ARABIC_LETTER_ALEF_MAKSOORA, ""+ARABIC_LETTER_YEH));
		ALTERNATIVE_ENDINGS.add(new Pair(""+ARABIC_LETTER_YEH, ""+ARABIC_LETTER_ALEF_MAKSOORA));
		ALTERNATIVE_ENDINGS.add(new Pair(""+ARABIC_LETTER_HEH, ""+ARABIC_LETTER_TEH_MARBUTA));
		ALTERNATIVE_ENDINGS.add(new Pair(""+ARABIC_LETTER_TEH_MARBUTA, ""+ARABIC_LETTER_HEH));
		ALTERNATIVE_ENDINGS.add(new Pair(""+ARABIC_LETTER_ALEF_MAKSOORA+ARABIC_LETTER_HAMZA, ""+ARABIC_LETTER_YEH_WITH_HAMZA_ABOVE));
		ALTERNATIVE_ENDINGS.add(new Pair(""+ARABIC_LETTER_YEH+ARABIC_LETTER_HAMZA, ""+ARABIC_LETTER_YEH_WITH_HAMZA_ABOVE));
		ALTERNATIVE_ENDINGS.add(new Pair(""+ARABIC_LETTER_YEH_WITH_HAMZA_ABOVE, ""+ARABIC_LETTER_YEH+ARABIC_LETTER_HAMZA));
	}

	/** Constructs an arabic morphological analyzer
	 * @param outputStream The stream where to output the results. Can be <CODE>null</CODE> if no output is
	 * desired
	 * @param verbose Whether or not the analyzer should output some convenience messages
	 */
	public AraMorph(DictionaryHandler dictionaryHandler, SolutionsHandler solutionsHandler) {
		dict = dictionaryHandler;
		sol = solutionsHandler;
		logger.info("Initializing in-memory dictionary handler...");
	}
	
	/**
	 * Retrieves an instance that has its dictionaries normalized and uses JDBC
	 * @return an instance of AraMorph
	 */
	public static AraMorph getNormalizedJDBC() {
		if (NORMALIZED_JDBC_ARAMORPH==null) {
			NORMALIZED_JDBC_ARAMORPH = getDefault(true, true);
		}
		return NORMALIZED_JDBC_ARAMORPH;
	}
	
	private static Map<Integer, AraMorph> defaultInstances = new HashMap<Integer, AraMorph>();
	
	public static AraMorph getDefault(boolean dictionaryNormalized, boolean useJDBC) {
		int key = 0;
		if (dictionaryNormalized)
			key |= 2;
		if (useJDBC)
			key |= 1;
		AraMorph araMorph = defaultInstances.get(key);
		if (araMorph == null) {
			araMorph = createDefault(dictionaryNormalized, useJDBC);
			defaultInstances.put(key, araMorph);
		}
		return araMorph;
	}
	
	/**
	 * Returns the default AraMorph to be used.
	 * @param dictionaryNormalized whether the dictionary normalized or not
	 * @return a default AraMorph instance
	 */
	private static AraMorph createDefault(boolean dictionaryNormalized, boolean useJDBC) {
		Dictionary prefixes, stems, suffixes;
		CompatibilityTable tableAB, tableAC, tableBC;
		if (useJDBC) {
			prefixes = new JDBCDictionary("DICT_PREFIXES",dictionaryNormalized);
			stems = new JDBCDictionary("DICT_STEMS",dictionaryNormalized);
			suffixes = new JDBCDictionary("DICT_SUFFIXES",dictionaryNormalized);
			//load 3 compatibility tables
			InMemoryCompatibilityTable table;
			table = new InMemoryCompatibilityTable("TABLE_AB");
			table.loadFromDB(DBManager.readonlyConnection);
			tableAB = table;
			table = new InMemoryCompatibilityTable("TABLE_AC");
			table.loadFromDB(DBManager.readonlyConnection);
			tableAC = table;
			table = new InMemoryCompatibilityTable("TABLE_BC");
			table.loadFromDB(DBManager.readonlyConnection);
			tableBC = table;
		} else {
			// load 3 lexicons
			InMemoryDictionary temp;
			temp = new InMemoryDictionary(dictionaryNormalized);
			temp.loadFromXML(AraMorph.class.getResourceAsStream("/dictionaries/dictPrefixes.xml"));
			prefixes = temp;
			
			temp = new InMemoryDictionary(dictionaryNormalized);
			temp.loadFromXML(AraMorph.class.getResourceAsStream("/dictionaries/dictStems.xml"));
			stems = temp;
			
			temp = new InMemoryDictionary(dictionaryNormalized);
			temp.loadFromXML(AraMorph.class.getResourceAsStream("/dictionaries/dictSuffixes.xml"));
			suffixes = temp;
			//load 3 compatibility tables
			InMemoryCompatibilityTable temp2;
			temp2 = new InMemoryCompatibilityTable("TABLE_AB");
			temp2.loadFromXML(AraMorph.class.getResourceAsStream("/dictionaries/tableAB.xml"));
			tableAB = temp2;
			
			temp2 = new InMemoryCompatibilityTable("TABLE_AC");
			temp2.loadFromXML(AraMorph.class.getResourceAsStream("/dictionaries/tableAC.xml"));
			tableAC = temp2;
			
			temp2 = new InMemoryCompatibilityTable("TABLE_BC");
			temp2.loadFromXML(AraMorph.class.getResourceAsStream("/dictionaries/tableBC.xml"));
			tableBC = temp2;
		}
		
		return new AraMorph(new DictionaryHandler(prefixes,stems,suffixes,tableAB,tableAC,tableBC),new InMemorySolutionsHandler(FOUND_LIMIT));
	}
	
	/** Returns a word in the Buckwalter transliteration system from a word in arabic.
	 * Vowels and diacritics are <strong>discarded</strong>.
	 * @param word The word in arabic
	 * @return The romanized word
	 */
	public static synchronized String romanizeWord(String word) {
		return StringOperations.translate(word,ARABIC2ROMAN);
	}
	
	/** Return an word in arabic from a word in the Buckwalter transliteration system.
	 * @param translitered The romanized word
	 * @return The word in arabic
	 */
	public static synchronized String arabizeWord(String translitered) {
		return StringOperations.translate(translitered,ROMAN2ARABIC);
	}
	
	/** Feed an internal list of solutions for the given word
	 * @param word The word. It is assumed that {@link #romanizeWord(String word) romanizeWord} has been called before
	 * @return Whether or not there are solutions for this word
	 */
	private Collection<Solution> getDirectWordSolutions(String word) {
		Collection<Solution> wordSolutions = new LinkedList<Solution>();
		int cnt = 0;
		int suffixStart = Math.max(word.length()-6,1);
		//get a list of valid segmentations
		//1- try all possible prefixes
		for (int i=Math.min(4,word.length()-1);i>=0;i--) {
			//extract the prefix from the word
			String strPrefix = word.substring(0,i);
			//check whether this prefix exists or not
			if (dict.hasPrefix(strPrefix)) {
				//2- try all possible suffixes
				for (int j=Math.max(suffixStart,i+1);j<=word.length();j++) {
					//extract the suffix from the word
					String strSuffix = word.substring(j);
					//extract the stem from the word
					String strStem = word.substring(i,j);
					//check whether this suffix and stem exist or not 
					if (dict.hasSuffix(strSuffix) && dict.hasStem(strStem)) {
						//iterate through all prefixes
						for (Iterator<DictionaryEntry> itPrefix=dict.getPrefixIterator(strPrefix);itPrefix.hasNext();) {
							DictionaryEntry prefix=itPrefix.next();
							//iterate through all suffixes
							for (Iterator<DictionaryEntry> itSuffix=dict.getSuffixIterator(strSuffix);itSuffix.hasNext();) {
								DictionaryEntry suffix=itSuffix.next();
								//check whether the prefix is compatible with the suffix or not
								if (dict.hasAC(prefix.getMorphology(),suffix.getMorphology())) {
									//iterate through all stems
									for (Iterator<DictionaryEntry> itStem=dict.getStemIterator(strStem);itStem.hasNext();) {
										DictionaryEntry stem=itStem.next();
										//check whether the stem is compatible with prefix and suffix
										if (dict.hasAB(prefix.getMorphology(),stem.getMorphology()) &&
												dict.hasBC(stem.getMorphology(),suffix.getMorphology()))
											wordSolutions.add(new Solution(++cnt, prefix, stem, suffix));
									}
								}
							}
						}
					}
				}
			}
		}
		if (wordSolutions.isEmpty())
			wordSolutions = null;
		return wordSolutions;
	}
	
	/** Feed an internal list of alternative spellings for the given word
	 * @param originalWord The word. It is assumed that {@link #romanizeWord(String word) romanizeWord} has been called before
	 * @return Whether or not there are alternative spellings for this word
	 */
	private Iterator<String> getAlternativeSpellings(String originalWord) {
		//check for cached values
		Iterator<String> alternativeSpell = sol.getAlternativeSpellingsIterator(originalWord);
		if (alternativeSpell != null)
			return alternativeSpell;
		//this will hold all possible spellings for the given word
		LinkedList<String> wordAlternativeSpellings = new LinkedList<String>();
		wordAlternativeSpellings.add(originalWord);
		//first change the alternations in the word
		for (Pair p : ALTERNATIVE_SPELLINGS) {
			int i = originalWord.length();
			while ((i=originalWord.substring(0,i).lastIndexOf(p.first))!=-1) {
				Collection<String> temp = new LinkedList<String>();
				for (String w : wordAlternativeSpellings) {
					temp.add(w.substring(0,i)+p.second+w.substring(i+p.first.length()));
				}
				wordAlternativeSpellings.addAll(temp);
			}
		}
		
		//scond change the alternations at the end of the word
		for (Pair p : ALTERNATIVE_ENDINGS) {
			if (originalWord.endsWith(p.first)) {
				Collection<String> temp = new LinkedList<String>();
				for (String w : wordAlternativeSpellings) {
					temp.add(w.substring(0,w.length()-p.first.length())+p.second);
				}
				wordAlternativeSpellings.addAll(temp);
			}
		}
		
		//remove original word from the set of alternative spellings
		//as it has been already processed
		wordAlternativeSpellings.removeFirst();
		if (wordAlternativeSpellings.size() > 0) {
			sol.addAlternativeSpellings(originalWord,wordAlternativeSpellings);
			return wordAlternativeSpellings.iterator();
		}
		return null;
	}
	
	/** Returns the solutions for a previously analyzed word.
	 * @param word The word.
	 * @return A list of solutions
	 * @see Solution
	 */
	public Iterator<Solution> getWordSolutions(String word) {
		logger.fine("Processing token : " + "\t" + word);
		//check for cached value
		Iterator<Solution> solutions = sol.getSolutionsIterator(word);
		if (solutions != null)
			return solutions;
		//TODO : check accuracy
		//check that token is an Arabic word
		int i=0;
		if (word.length()>0)
			while (i<word.length()&&ARABIC_LETTERS.contains(new Integer(word.charAt(i))))
				i++;
		if (i < word.length()) {
			word = word.trim();
			// tokenize it on white space
			String subTokens[] = word.split("\\s+");
			for (i = 0 ; i < subTokens.length ; i++) {
				if (!"".equals(subTokens[i])) {
					nonArabicTokensCounter++;
					logger.info("Non-Arabic : " + subTokens[i]);
				}
			}
			return null;
		}
		arabicTokensCounter++;
		//word has solutions...
		//try to find direct solutions
		Collection<Solution> wordSolutions;
		wordSolutions = getDirectWordSolutions(word);

		//no direct solutions, check for alternative spelling
		if (wordSolutions == null) {
			Iterator<String> alternatives = getAlternativeSpellings(word);
			if (alternatives != null) {
				wordSolutions = new LinkedList<Solution>();
				//word has no direct solution
				//if there are some alternative spellings
				
				while (alternatives.hasNext()) {
					String alternative = alternatives.next();
					//feed solutions with alternative spellings' ones
					Collection<Solution> alternativeSolutions = getDirectWordSolutions(alternative);
					if (alternativeSolutions != null)
						wordSolutions.addAll(alternativeSolutions);
				}
				if (wordSolutions.isEmpty())
					wordSolutions = null;
			}		
		}
		if (wordSolutions != null) {
			sol.addSolutions(word, wordSolutions);
			return wordSolutions.iterator();
		}
		return null;
	}
	
	/** Display the statistics on what has been analyzed so far. */
	public void printStats() {
		DecimalFormat df = new DecimalFormat("##.##%");
		double total = found.size() + notFound.size(); //double to force casting
		logger.info("=================== Statistics ===================");
		logger.info("Lines : " + linesCounter);
		logger.info("Arabic tokens : " + arabicTokensCounter);
		logger.info("Non-arabic tokens : " + nonArabicTokensCounter);
		logger.info("Words found : " + found.size() + " (" + df.format(found.size() / total) + ")");
		logger.info("Words not found : " + notFound.size() + " (" + df.format(notFound.size() / total) + ")");
		logger.info("==================================================");
	}
	
	/** Display help for command line interface. */
	private static void PrintUsage() {
		logger.warning("Arabic Morphological Analyzer for Java(tm)");
		logger.warning("Ported to Java(tm) by Pierrick Brihaye, 2003-2004.");
		logger.warning("Based on :");
		logger.warning("BUCKWALTER ARABIC MORPHOLOGICAL ANALYZER");
		logger.warning("Portions (c) 2002 QAMUS LLC (www.qamus.org),");
		logger.warning("(c) 2002 Trustees of the University of Pennsylvania.");
		logger.warning("This program is governed by :");
		logger.warning("The GNU General Public License");		
		logger.warning("");
		logger.warning("Usage :");
		logger.warning("");
		logger.warning("araMorph inFile [inEncoding] [outFile] [outEncoding] [-v]");
		logger.warning("");
		logger.warning("inFile : file to be analyzed");
		logger.warning("inEncoding : encoding for inFile, default CP1256");
		logger.warning("outFile : result file, default console");
		logger.warning("outEncoding : encoding for outFile, if not specified use Buckwalter transliteration with system's file.encoding");
		logger.warning("-v : verbose mode");
	}
	
	/**
	 * Translates the given word to the other language (Arabic <-> English).
	 * @param word the word to be translated. It will be checked if it is Arabic or English
	 * @param firstParts the input word after it is analyzed and separated into parts. Only for Arabic input till now
	 * @param translations the translation of each part in parts.
	 * @return size of the arrays parts and translations
	 */
	public int translate(String word, Collection<DictionaryEntry> firstParts, Map<DictionaryEntry, Collection<DictionaryEntry>> parts) {
		word = word.toLowerCase();
		int nSolutions=0;
		if (word.matches("[a-z\\ ]*")) {
			//english word, translate into arabic
			for (Iterator<DictionaryEntry> iter = dict.getStemIteratorByGloss(word); iter.hasNext();) {
				firstParts.add(iter.next());
				nSolutions++;
			}
		} else if (getWordSolutions(word) != null) {
			//Arabic word and has solutions
			//translate into English
			Iterator<Solution> i = getWordSolutions(word);
			while (i != null && i.hasNext()) {
				Solution sol = i.next();
				firstParts.add(sol.getPrefix());
				
				Collection<DictionaryEntry> coll;
				coll = parts.get(sol.getPrefix());
				if (coll == null) {
					coll = new ArrayList<DictionaryEntry>();
					parts.put(sol.getPrefix(),coll);
				}
				coll.add(sol.getStem());
				
				coll = parts.get(sol.getStem());
				if (coll == null) {
					coll = new ArrayList<DictionaryEntry>();
					parts.put(sol.getStem(),coll);
				}
				coll.add(sol.getSuffix());
				
				nSolutions++;
			}
		}
		return nSolutions;
	}
	
	/** Entry point for command line interface.
	 * @param args Arguments
	 */
	public static void main(String[] args) {
		
		boolean argsOK = true;
		String inputFile = null;
		String outputFile = null;
		String inputEncoding = null;
		String outputEncoding = null;
		Boolean verbose = null;
				
		if (args.length == 0) argsOK = false;
		//TODO : should we be more severe ?
		else {
			for (int i = 0 ; i < args.length ; i++) {
				//verbose flag ?
				if ("-v".equals(args[i])) {
					if (verbose == null) {
						verbose = new Boolean(true);
						continue;
					}
					argsOK = false;
					break;
				}
				//is it a charset ?
				try {
					Charset.forName(args[i]);
					if (inputEncoding == null) {
						inputEncoding = args[i];
						continue;
					}
					else if (outputEncoding == null) {
						outputEncoding = args[i];
						continue;
					}
					//too many charsets
					else {
						argsOK = false;
						break;
					}
				}
				catch (IllegalCharsetNameException e) {}
				catch (UnsupportedCharsetException e) {}
				//is it a file name ?
				if (inputFile == null) {
					inputFile = args[i];
					continue;
				}
				else if (outputFile == null) {
					outputFile = args[i];
					continue;
				}
				//too many files
				else {
					argsOK = false;
					break;
				}
			}
		}
		
		if (!argsOK || inputFile == null) PrintUsage();
		else {
			
			if (inputEncoding == null) inputEncoding = "Cp1256"; //TODO : change default ?						
			if (verbose == null) verbose = new Boolean(false);
			LineNumberReader IN = null;
			PrintStream ps = null;
			
			if (outputFile != null) {
				try {
					if (outputEncoding == null)
						ps = new PrintStream(new FileOutputStream(outputFile), true);
					else
						ps = new PrintStream(new FileOutputStream(outputFile), true, outputEncoding);
				} catch (FileNotFoundException e) {
					logger.warning("Can't write to output file : " + outputFile);
				} catch (UnsupportedEncodingException e) {
					logger.warning("Unsupported output encoding : " + outputEncoding);
				}
			}
			else ps = System.out;
			
			try {
				//set the logging options for AraMorph (as I am not a member of it)
				Logger logger = Logger.getLogger(AraMorph.class.getName());
				logger.setLevel(verbose.booleanValue()?Level.FINE:Level.INFO);
				for (Handler handler : logger.getHandlers())
					logger.removeHandler(handler);
				logger.addHandler(new StreamHandler(ps,new SimpleFormatter()));
				//Construct an instance AraMorph
				AraMorph araMorph = AraMorph.getNormalizedJDBC();			
				IN = new LineNumberReader(new BufferedReader(new InputStreamReader(new FileInputStream(inputFile),inputEncoding)));
				//XXX: araMorph.getWordSolutions(IN);
				araMorph.printStats();
			}
			catch (IOException e) {
				throw new RuntimeException("Problem : " + e.getMessage());
			}
			finally {
				try {
					if (IN != null) IN.close();
				}
				catch (IOException e) {}
				if (ps != null) ps.close();
			}
		}
	}
}
